#!/usr/bin/env python
# -*- coding:utf-8 -*-
# @FileName  :获取章节链接.py
# @Time      :2024/5/10 
# @Author    :CL
# @email     :1037654919@qq.com

# 获取章节链接
import json
import os
import re
import time

import requests
from bs4 import BeautifulSoup
from util import mongo_manager
ximalaya_sound = mongo_manager('ximalaya_sound',db = 'public_data')

headers = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
    "Accept-Language": "zh-CN,zh;q=0.9",
    "Cache-Control": "no-cache",
    "Connection": "keep-alive",
    "Pragma": "no-cache",
    "Sec-Fetch-Dest": "document",
    "Sec-Fetch-Mode": "navigate",
    "Sec-Fetch-Site": "none",
    "Sec-Fetch-User": "?1",
    "Upgrade-Insecure-Requests": "1",
    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
    "sec-ch-ua": "\"Chromium\";v=\"124\", \"Google Chrome\";v=\"124\", \"Not-A.Brand\";v=\"99\"",
    "sec-ch-ua-mobile": "?0",
    "sec-ch-ua-platform": "\"Linux\""
}
cookies = {
    "_xmLog": "h5&2077b7d4-cefb-43c2-88d4-0c0e6ded2d45&process.env.sdkVersion",
    "wfp": "ACNhYmU2MmY1MWVhOWYzOWIzeyc-g_H7XL94bXdlYl93d3c",
    "xm-page-viewid": "ximalaya-web",
    "impl": "www.ximalaya.com.login",
    "x_xmly_traffic": "utm_source%253A%2526utm_medium%253A%2526utm_campaign%253A%2526utm_content%253A%2526utm_term%253A%2526utm_from%253A",
    "Hm_lvt_4a7d8ec50cfd6af753c4f8aee3425070": "1713495863,1713508064,1713508064",
    "Hm_lpvt_4a7d8ec50cfd6af753c4f8aee3425070": "1713512092"
}
def get_sound_list(_type = '百科',album_url = "https://www.ximalaya.com/album/62755117"):
    headers = {
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
        "Accept-Language": "zh-CN,zh;q=0.9",
        "Cache-Control": "no-cache",
        "Connection": "keep-alive",
        "Pragma": "no-cache",
        "Sec-Fetch-Dest": "document",
        "Sec-Fetch-Mode": "navigate",
        "Sec-Fetch-Site": "none",
        "Sec-Fetch-User": "?1",
        "Upgrade-Insecure-Requests": "1",
        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
        "sec-ch-ua": "\"Chromium\";v=\"124\", \"Google Chrome\";v=\"124\", \"Not-A.Brand\";v=\"99\"",
        "sec-ch-ua-mobile": "?0",
        "sec-ch-ua-platform": "\"Linux\""
    }
    cookies = {
        "_xmLog": "h5&2077b7d4-cefb-43c2-88d4-0c0e6ded2d45&process.env.sdkVersion",
        "wfp": "ACNhYmU2MmY1MWVhOWYzOWIzeyc-g_H7XL94bXdlYl93d3c",
        "xm-page-viewid": "ximalaya-web",
        "impl": "www.ximalaya.com.login",
        "x_xmly_traffic": "utm_source%253A%2526utm_medium%253A%2526utm_campaign%253A%2526utm_content%253A%2526utm_term%253A%2526utm_from%253A",
        "Hm_lvt_4a7d8ec50cfd6af753c4f8aee3425070": "1713495863,1713508064,1713508064",
        "Hm_lpvt_4a7d8ec50cfd6af753c4f8aee3425070": "1713946134"
    }
    response = requests.get(album_url, headers=headers, cookies=cookies)
    print(response.url,response)
    if response.status_code == 200:
        soup = BeautifulSoup(response.text, 'html.parser')
        try:
            category = soup.find('div', class_='bread-crumb xV_').findAll('a')[1].text
        except:
            category = ''
        try:
            h2 = soup.find('h2', class_='bread-crumb-link xV_').text
        except:
            h2 = ''
        print(category,h2)
        try:
            # 定义正则表达式模式
            pattern = r'<script>\s*window\.__INITIAL_STATE__\s*=\s*({.*?});\s*</script>'
            # 使用 re 模块进行匹配
            match = re.search(pattern, response.text)

            # 如果找到匹配项
            if match:
                # 提取匹配到的内容
                initial_state_json = match.group(1)
                json_obj = json.loads(initial_state_json)
                tracks =json_obj['store']['AlbumDetailTrackListV2']['tracksInfo']['tracks']
                # datas = soup.find('div', class_='sound-list  H_g').findAll('li',class_='_nO')  # head H_g
                for track in tracks:
                    title = track['title']
                    href = track['url']
                    duration = track['duration']
                    print(category,h2,title, href)
                    if 'https://www.ximalaya.com' in href:
                        sound_url = href
                    else:
                        sound_url = 'https://www.ximalaya.com'+href
                    result = {'_type':_type,"_id":sound_url,'category': category, 'h2': h2, 'title': title, 'href': sound_url,'duration':duration}
                    result['album_url'] = album_url
                    try:
                        ximalaya_sound.insertOne(result)
                    except Exception as e:
                        ximalaya_sound.updateOne({"_id": sound_url}, result)
                #seo-track-list H_g 数据
                seoTrackList = json_obj['store']['AlbumDetailTrackListV2']['seoTrackList']
                for track in seoTrackList:
                    title = track['trackName']
                    href = track['uri']
                    print(category,h2,title, href)
                    if 'https://www.ximalaya.com' in href:
                        sound_url = href
                    else:
                        sound_url = 'https://www.ximalaya.com'+href
                    result = {'_type':_type,"_id":sound_url,'category': category, 'h2': h2, 'title': title, 'href': sound_url}
                    result['album_url'] = album_url
                    try:
                        ximalaya_sound.insertOne(result)
                    except Exception as e:
                        ximalaya_sound.updateOne({"_id": sound_url}, result)
        except:
            print('没有 __INITIAL_STATE__ 数据')
        # seo-track-list H_g 数据的补充
        try:
            datas = soup.find('ul', class_='seo-track-list H_g').findAll('li')
            for l in datas:
                title = l.find('a', class_='b_t').text
                href = l.find('a', class_='b_t')['to']
                print(category,h2,title, href)
                if 'https://www.ximalaya.com' in href:
                    sound_url = href
                else:
                    sound_url = 'https://www.ximalaya.com' + href
                result = {'_type':_type,"_id":sound_url,'category': category, 'h2': h2, 'title': title, 'href': sound_url}
                result['album_url'] = album_url
                try:
                    ximalaya_sound.insertOne(result)
                except Exception as e:
                    print(e)
        except:
            pass

def export_sound_list(seed):
    sound_url = seed['href']
    response = requests.get(sound_url, headers=headers, cookies=cookies)

    print(response.url,response)
    if response.status_code == 200:
        soup = BeautifulSoup(response.text, 'html.parser')
        try:
            datas = soup.find('ul', class_='sound-list type o_O').findAll('li')
            print(len(datas))
            if datas:
                for l in datas:
                    title = l.find('a').text
                    href = l.find('a')['href']
                    if 'https://www.ximalaya.com' in href:
                        sound_url = href
                    else:
                        sound_url = 'https://www.ximalaya.com' + href
                    result = {}
                    if '_type' in seed:
                        result['_type'] = seed['_type']
                    if 'category' in seed:
                        result['category'] = seed['category']
                    if 'h2' in seed:
                        result['h2'] = seed['h2']
                    if 'album_url' in seed:
                        result['album_url'] = seed['album_url']
                    result.update( {"_id": sound_url,  'title': title, 'href': sound_url})
                    try:
                        ximalaya_sound.insertOne(result)
                        print(title, sound_url,'insert')
                    except:
                        # print(title, sound_url,'插入失败')
                        # ximalaya_sound.updateOne({"_id": sound_url}, result)
                        pass
            else:
                print('no find  sound-list type o_O datas')
        except Exception as e :
            print(e,'no find  sound-list type o_O')
def main1(): # 获取所有相关联的音频链接
    while True:
        seed = ximalaya_sound.find_one_and_update({"status_findmore": None}, {"status_findmore": 'running'})
        if seed:
            export_sound_list(seed)
            seed['status_findmore'] = 'done'
            ximalaya_sound.updateOne({"_id": seed['_id']}, seed)
        else:
            break
#基于album链接获取sound链接
def main():
    dicts = {
        '小说': [
            # 'https://www.ximalaya.com/album/5822114',
            #      'https://www.ximalaya.com/album/33004172',
                 'https://www.ximalaya.com/album/46595930',
                 'https://www.ximalaya.com/album/32608127',
                 'https://www.ximalaya.com/album/55722180',
                 'https://www.ximalaya.com/album/50313104',
                 'https://www.ximalaya.com/album/72776436',
                 'https://www.ximalaya.com/album/72284380',
                 # 'https://www.ximalaya.com/album/47405490',
                 'https://www.ximalaya.com/album/69404265',
                 'https://www.ximalaya.com/album/46595930'],
        '普通话':[
            # 'https://www.ximalaya.com/album/46466082',
            #  'https://www.ximalaya.com/album/50552092',
            #  'https://www.ximalaya.com/album/21328128',
            #  'https://www.ximalaya.com/album/2808888'
        ],
        '美食': ['https://www.ximalaya.com/album/69921452',
                 'https://www.ximalaya.com/album/31827070',
                 'https://www.ximalaya.com/album/18551086'],
        '娱乐': ['https://www.ximalaya.com/album/32712303',
                 'https://www.ximalaya.com/album/5879804'],
        '百科': ['https://www.ximalaya.com/album/62755117']
    }
    for key, lists in dicts.items():

        for album_url in lists:
            print(key, album_url)
            get_sound_list(_type=key, album_url=album_url)
#基于sound_url 获取声音简介（即文本）
def get_txt_by_sound_url():
    #含有文本的音频 名字
    lists = ['纯声音无配乐 哲思散文：一个人的村庄', '普通话水平测试短文60篇（有字幕、无背景音）',
             '普通话测试朗读60篇（有字幕）', '普通话 | 命题说话90篇(字幕)', '道德经｜无废话无音乐有文字版',
             '易传（ 含字幕）', '冷血法医', '普通话60篇国家级示范与打卡（有字幕拼音）']
    album_list =['https://www.ximalaya.com/album/47405490',
'https://www.ximalaya.com/album/46466082',
'https://www.ximalaya.com/album/50552092',
'https://www.ximalaya.com/album/21328128',
'https://www.ximalaya.com/album/2808888',
'https://www.ximalaya.com/album/5822114',
'https://www.ximalaya.com/album/33004172']
    for album_url in album_list:
        print(album_url)
        while True:
            seed = ximalaya_sound.find_one_and_update({"album_url": album_url,'status_text':None}, {"status_text": 'running'})
            if seed:
                sound_url = seed['_id']
                try:
                    h2 = seed["h2"]
                    _type = seed["_type"]
                    name = str(seed["title"]).replace("/", "_")
                    path = os.path.join("/home/chenglei3/work/data/ximalaya_text/", _type, h2)   #ximalaya_text
                    os.makedirs(path, exist_ok=True)
                    if 'text' in seed:
                        article = seed['text']
                        with open(f'{path}/{name}.txt', 'a', encoding='utf-8') as f:
                            f.write(article)
                    else:
                        res = requests.get(sound_url, headers=headers, timeout=10)
                        soup = BeautifulSoup(res.text, 'lxml')
                        # 获取音频简介
                        article = soup.find('article', class_='intro').text
                        # print(article)
                        with open(f'{path}/{name}.txt', 'a', encoding='utf-8') as f:
                            f.write(article)

                    ximalaya_sound.updateOne({"_id": seed['_id']}, {'status_text': 'done',"text":article})
                except Exception as e:
                    print(seed['_id'],e)
                    time.sleep(10)
                    ximalaya_sound.updateOne({"_id": seed['_id']}, {'status_text': 'error'})
            else:
                break
            # break #临时


if __name__ == '__main__':
    print()
    # 数据库维护
    # ximalaya_sound.updateMany({"_type": '娱乐'}, {"status_findmore": None})

    # # 基于album链接获取sound链接
    # main()
    #
    # # 基于sound_url获取更多的sound_url
    # main1()

    # 基于sound_url 获取声音简介（即文本）
    ximalaya_sound.updateMany({"status_text": 'running'}, {"status_text": None})
    get_txt_by_sound_url()

    ximalaya_sound.close()
